package com.atlassian.uwc.ui;

import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.TreeSet;
import java.util.Vector;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.activation.MimetypesFileTypeMap;

import org.apache.log4j.Logger;
import org.apache.xmlrpc.XmlRpcException;

import biz.artemis.confluence.xmlrpcwrapper.AttachmentForXmlRpc;
import biz.artemis.confluence.xmlrpcwrapper.BlogForXmlRpc;
import biz.artemis.confluence.xmlrpcwrapper.CommentForXmlRpc;
import biz.artemis.confluence.xmlrpcwrapper.ConfluenceServerSettings;
import biz.artemis.confluence.xmlrpcwrapper.PageForXmlRpc;
import biz.artemis.confluence.xmlrpcwrapper.RemoteWikiBroker;
import biz.artemis.confluence.xmlrpcwrapper.SpaceForXmlRpc;
import biz.artemis.confluence.xmlrpcwrapper.SpaceForXmlRpc.SpaceType;

import com.atlassian.uwc.converters.Converter;
import com.atlassian.uwc.converters.IllegalLinkNameConverter;
import com.atlassian.uwc.converters.IllegalPageNameConverter;
import com.atlassian.uwc.converters.JavaRegexConverter;
import com.atlassian.uwc.converters.PerlConverter;
import com.atlassian.uwc.converters.twiki.JavaRegexAndTokenizerConverter;
import com.atlassian.uwc.converters.twiki.TWikiRegexConverterCleanerWrapper;
import com.atlassian.uwc.converters.xml.DefaultXmlEvents;
import com.atlassian.uwc.converters.xml.XmlEvents;
import com.atlassian.uwc.filters.FilterChain;
import com.atlassian.uwc.hierarchies.HierarchyBuilder;
import com.atlassian.uwc.hierarchies.HierarchyNode;
import com.atlassian.uwc.splitters.PageSplitter;
import com.atlassian.uwc.ui.listeners.FeedbackHandler;
import com.atlassian.uwc.ui.listeners.TestSettingsListener;

/**
 * This class drives the conversion process by gathering all the files, gathering
 * the selected converters, then applying all the converters against each file, then
 * sending the converted 'Pages' and attachments to Confluence via XML-RPC (or
 * possibly some other method in the future)
 */
public class ConverterEngine implements FeedbackHandler {
    
	private static final String DEFAULT_ATTACHMENT_UPLOAD_COMMENT = "Added by UWC, the Universal Wiki Converter";
	/* START CONSTANTS */
	private static final int NUM_REQ_CONVERTERS = 2;
	private static final String REQUIRED_CONVERTER_ILLEGAL_LINKS = "MyWiki.9999.illegal-links.class=com.atlassian.uwc.converters.IllegalLinkNameConverter";
	private static final String REQUIRED_CONVERTER_ILLEGAL_NAMES = "MyWiki.9999.illegal-names.class=com.atlassian.uwc.converters.IllegalPageNameConverter";
	private static final String NONCONVERTERTYPE_PAGEHISTORYPRESERVATION = "page-history-preservation";
	private static final String NONCONVERTERTYPE_HIERARCHYBUILDER = ".hierarchy-builder";
	private static final String NONCONVERTERTYPE_ILLEGALHANDLING = "illegal-handling";
	private static final String NONCONVERTERTYPE_AUTODETECTSPACEKEYS = "autodetect-spacekeys";
	private static final String NONCONVERTERTYPE_MISCPROPERTIES = ".property";
	private static final String NONCONVERTERTYPE_FILTERS = ".filter";
	private static final String NONCONVERTERTYPE_XMLEVENT = ".xmlevent";
	private static final String CONVERTERTYPE_TWIKICLEANER = ".twiki-cleaner";
	private static final String CONVERTERTYPE_JAVAREGEX = ".java-regex";
	private static final String CONVERTERTYPE_JAVAREGEXTOKEN = ".java-regex-tokenize";
	private static final String CONVERTERTYPE_PERL = ".perl";
	private static final String CONVERTERTYPE_CLASS = ".class";

	private static final String XMLEVENT_PROP_ERROR = "Xmlevent Property must follow this format convention: {tag}xmltag{class}com.something.Class";

	private static final String PROP_ATTACHMENT_SIZE_MAX = "attachment.size.max";
	private static final int DEFAULT_NUM_STEPS = 1000;
	private static final String ORPHAN_ATTACHMENTS_PAGE_TITLE="Orphan attachments";
	
	
	/* START FIELDS */
	public boolean running = false; //Methods check this to see if the conversion needs to be cancelled
	
	/**
	 * used to disable check for illegal names and links. 
	 * We want to allow users to override this so they can handle it themselves
	 * with converters. 
	 */
	private boolean illegalHandlingEnabled = true; //default = true
	private boolean autoDetectSpacekeys = false; //default = false
	private HashSet<String> attachedFiles;
	
    Logger log = Logger.getLogger(this.getClass());
    // this logger is used to write out totals for the UWC to a seperate file uwc-totals.log
    Logger totalsFileLog = Logger.getLogger("totalsFileLog");

    /**
     * The string that directory separators (e.g., / on Unix and \ on Windows) are replaced
     * with in page titles.
     * This is used by DokuWikiLinkConverter too.
     */
    public static final String CONFLUENCE_SEPARATOR = " -- ";

    protected enum HierarchyHandler {
    	DEFAULT, 			//no hierarchy handling
    	HIERARCHY_BUILDER,	//hierarchyBuilder handles 
    	PAGENAME_HIERARCHIES//hierarchy maintained in pagename
    }
    private HierarchyHandler hierarchyHandler = HierarchyHandler.DEFAULT; 
    
    /**
     * The mapping from file name extension to mime type that is used when sending
     * attachments to Confluence.
     * NOTE: static so that other files can get access to this easily.
     */
    private static MimetypesFileTypeMap mimeTypes;

    /**
     * This is the location of the mime type mapping file. For details on the file format,
     * refer to the link below.
     *
     * @see javax.activation.MimetypesFileTypeMap
     */
    public final static String mimetypeFileLoc = "conf" + File.separator + "mime.types";

    /**
     * This field is set if a hierarchy builder "converter" is used. The field controls the
     * way in which pages are added/updated in Confluence. If hierarchyBuilder is <code>null</code>, all
     * pages are added as top-level pages in the selected space. Otherwise, the hierarchy builder is
     * called on to create a page hierarchy, and the engine will insert the pages correspondingly.
     */
    private HierarchyBuilder hierarchyBuilder = null;
	private UWCUserSettings settings;
	private State state;
	private Properties miscProperties = new Properties(); //instantiate this here - UWC-293
	private Set<String> filterValues;

	/**
	 * the number of properties that are not converters from the properties file. 
	 * When we set up the progress bar, we calculate the max number of steps
	 * we're going to encounter with the number of properties that could be converters.
	 * But we update the progress bar a step only foreach converter property. So, we'll use
	 * this field to update the progress bar the extra amount.
	 */
	private int numNonConverterProperties;
	private Feedback feedback;

	private int newNodes;

    HashMap<String, Converter> converterCacheMap = new HashMap<String, Converter>();
	private long startTotalConvertTime;
	
	//Error handlers
	private ConverterErrors errors = new ConverterErrors();
	private boolean hadConverterErrors;
	private HashMap<String, String> homepages = new HashMap<String, String>();

	/* START CONSTRUCTORS */
    
    /**
     * This default constructor initializes the mime types.
     */
    public ConverterEngine() {
        try {
            mimeTypes = new MimetypesFileTypeMap(new FileInputStream(mimetypeFileLoc));
        } catch (FileNotFoundException e) {
            String note = "Couldn't load mime types!";
            log.error(note, e);
            this.errors.addError(Feedback.BAD_SETTINGS_FILE, note, false);
        }
        totalsFileLog.setAdditivity(false);
    }
    
    /* START METHODS */

    /**
     * converts the files with the converterstrings, and hooks any feedback into the given ui
     * @param inputPages pages from the filesystem to be converted
     * @param converterStrings list of converters as strings which will be run on the pages
     * @param sendToConfluence true if the pages should be uploaded to confluence
     * @param wikitype The wiki type that's being converted into Confluence, ex: Mediawiki
     */
    public void convert(List<File> inputPages, List<String> converterStrings, UWCUserSettings settings) {
    	//setup
    	this.running = true;
		resetFeedback();
		resetErrorHandlers();
		resetHierarchy();
    	
		//settings
		boolean sendToConfluence = Boolean.parseBoolean(settings.getSendToConfluence());
    	this.settings = settings;
    	if (!this.running) {
    		this.feedback = Feedback.CANCELLED;
    		return;
    	}
    	
    	//convert
		convert(inputPages, converterStrings, sendToConfluence, settings.getPattern());
		
		//cleanup
		if (this.feedback == Feedback.NONE)
			this.feedback = Feedback.OK;
		this.running = false;
    }
    
    /**
     * cancels the conversion
     */
    public void cancel() {
		String message = "Engine - Sending Cancel Signal";
    	log.debug(message);
		this.state.updateNote(message);
    	this.running = false;
    }

	/**
	 * gets a new State object.
	 * The State object will be used by the converter engine to measure progress.
	 * @param inputPages list of pages 
	 * @param converterStrings list of converters
	 * @param settings settings
	 * @return the state object the engine uses
	 */
	public State getState(List<File> inputPages, List<String> converterStrings, UWCUserSettings settings) {
		//The inputPages and converterString objects do not approximate the number of steps well because
		//inputPages includes directories which are counted as 1 object in the inputPages list, and
		//converterStrings includes non-converter properties. So, for now we'll use a default value to set up
		//the state.
//    	int steps = 
//    		getNumberOfSteps(
//    				inputPages, 
//    				converterStrings, 
//    				Boolean.parseBoolean(settings.getSendToConfluence())
//    				);
		return getState(settings);
	}

	/**
	 * gets a new State object, using a default number of steps.
	 * The State object will be used by the converter engine to measure progress.
	 * @param settings
	 * @return
	 */
	public State getState(UWCUserSettings settings) {
		int steps = DEFAULT_NUM_STEPS;
		String initialMessage = "Converting Wiki\n" +
				"Wikitype = " + settings.getWikitype() + "\n";
		this.state = new State(initialMessage, 0, steps);
		return state;
	}

	/**
	 * gets the number of steps for the given pages and converters lists
	 * @param pages list of objects representing pages. Can be any object: String, Page, etc.
	 * @param converters list of objects representing pages. Can be any object: String, Converter, etc.
	 * @param sendToConfluence true, if the conversion will upload the objects to confluence
	 * @return the number of steps that the engine will measure progress against
	 */
	protected int getNumberOfSteps(List pages, List converters, boolean sendToConfluence) {
		return getNumberOfSteps(pages.size(), converters.size(), sendToConfluence);
	}

	/**
	 * Counts the number of steps needed to do the entire conversion from start to finish.
	 * Here are the steps:
	 * 1. init converters - number of converters
	 * 2. create page objects - number of pages
	 * 3. convert the files - num of converterstrings * num of pages (or num of pages)
	 * 4. convert with required converters - num of pages
	 * 5. save pages - num of pages
	 * 6. upload pages (if send to confluence) num of pages, or 0
	 * @param pages number of pages
	 * @param converters number of converters
	 * @param sendToConfluence true if sending pages to confluence
	 * @return number of steps for performing conversion from start to finish
	 */
	private int getNumberOfSteps(int pages, int converters, boolean sendToConfluence) {
		return getNumberOfSteps(pages, converters, converters, sendToConfluence);
	}
	
	/**
	 * Counts the number of steps needed to do an entire conversion from start to finish.
	 * Used with progress monitor
	 * @param pages number of pages
	 * @param properties number of all converter file properties (including non-converter properties)
	 * @param converters number of converters
	 * @param sendToConfluence true if sending pages to Confluence
	 * @return number of steps for performing conversion from start to finish
	 */
	private int getNumberOfSteps(int pages, int properties, int converters, boolean sendToConfluence) {
		return getNumberOfSteps(pages, pages, properties, converters, sendToConfluence);
	}

	/**
	 * Counts the number of steps needed to do an entire conversion from start to finish.
	 * Used with progress monitor
	 * @param files number of files or directories chosen by the user. Does not count contents of directories seperately. 
	 * @param pages number of individual pages that will be converted
	 * @param properties number of all converter file properties (including non-converter properties)
	 * @param converters number of converters
	 * @param sendToConfluence true if sending pages to Confluence
	 * @return number of steps for performing conversion from start to finish
	 */
	private int getNumberOfSteps(int files, int pages, int properties, int converters, boolean sendToConfluence) {
		int numReqConverters = isIllegalHandlingEnabled()?NUM_REQ_CONVERTERS:0;
		int steps = 
			properties + 					//1. initialize converters (handles both converter and non-converter properties)
			files + 						//2. create page objects (uses the original list of chosen file objects)
			(converters * pages) +			//3. convert the files (uses the number of page objects)
			(numReqConverters) +			//4. create required converters (2, right now)
			(numReqConverters * pages) + 	//5. convert with required converters (2, right now)
			pages + 						//6. save the files
			(sendToConfluence?pages:0);		//7. upload pages if sendToConfluence
		return steps;
	}
	
	/**
	 * converts the given pages using the given converterStrings, and sends the pages
	 * to Confluence, if sendToConfluence is true.
	 * @param pages
	 * @param converterStrings
	 * @param sendToConfluence
	 */
	public void convert(List<File> pages, List<String> converterStrings, boolean sendToConfluence) {
		convert(pages, converterStrings, sendToConfluence, "");
	}

	/**
     * converts the given pages not filtered out with the given filterPattern
     * using the given converterStrings, and sends the pages to Confluence, 
     * if sendToConfluence is true
     * @param pages
     * @param converterStrings
     * @param sendToConfluence 
     * @param filterPattern ignores files with this filter pattern
     */
    public void convert(List<File> pages, List<String> converterStrings, boolean sendToConfluence, String filterPattern) {
    	log.info("Starting conversion.");
    	
    	initConversion();
    	
    	//create converters
    	ArrayList<Converter> converters = createConverters(converterStrings);

    	//create page objects - Recurse through directories, adding all files
    	FileFilter filter = createFilter(filterPattern);
    	List<Page> allPages = createPages(filter, pages);
    	
    	//fix progressbar max, which is dependent on the previous two lists
    	int steps = getNumberOfSteps(pages.size(), allPages.size(), converterStrings.size(), converters.size(), sendToConfluence);
    	this.state.updateMax(steps);
    	
    	
    	//convert the files
    	if (convertPages(allPages, converters)) {
    		//in case converting the pages disqualified some pages, we need to break if there are no pages left
    		if (allPages.size() < 1) {
    			String message = "All pages submitted were disqualified for various reasons. Could not complete conversion.";
				log.warn(message);
				this.errors.addError(Feedback.CONVERTER_ERROR, message, true);
				this.state.updateMax(this.state.getStep()); //complete progress bar, prematurely
				return;
    		}
    		//in case converting the pages disqualified some pages, we need to recompute progressbarmax
    		steps = getNumberOfSteps(pages.size(), allPages.size(), converterStrings.size(), converters.size(), sendToConfluence);
    		if (steps != this.state.getMax()) this.state.updateMax(steps);
    		
    		// do final required conversions. This step is seperate, due to state saving issues
    		convertWithRequiredConverters(allPages);

        	//save pages
			// TODO: This is not needed for the converter, and there should really be an option to turn it off.
			savePages(allPages, filterPattern);
			
			//check for namespace collisions and emit errors if found
			listCollisions(allPages);
			
			//handling page histories and not sorting on create
			if (isHandlingPageHistories() && 
					!(isPageHistorySortOnCreate())) {
				allPages = sortByHistory(allPages);
			}
			
	    	//upload pages, if the user approves
			if (sendToConfluence && this.running) {
				if (hierarchyHandler == HierarchyHandler.HIERARCHY_BUILDER && hierarchyBuilder != null) {
					//tell the hierarchy builder about the page histories framework
					//do this here so that we're sure the page histories properties are set
					if (hierarchyBuilder.getProperties() != null) { 
						hierarchyBuilder.getProperties().setProperty("switch."+NONCONVERTERTYPE_PAGEHISTORYPRESERVATION, isHandlingPageHistories()+"");
						if (getPageHistorySuffix() != null)	
							hierarchyBuilder.getProperties().setProperty("suffix."+NONCONVERTERTYPE_PAGEHISTORYPRESERVATION, getPageHistorySuffix());
					}
					//tell the hierarchy some other information
					if (hierarchyBuilder.getProperties() != null) {
						hierarchyBuilder.getProperties().setProperty("spacekey", settings.getSpace());
					}
					//build the hierarchy
					HierarchyNode root = hierarchyBuilder.buildHierarchy(allPages);
					writeHierarchy(root, allPages.size(), settings.getSpace());
				} else { //no hierarchy - just write the pages
					writePages(allPages, settings.getSpace());
				}
			}
			else if (!sendToConfluence){
				log.debug("Send To Confluence setting turned off. --> Not uploading pages.");
			}
    	}	
    	log.info("Conversion Complete");
    }

	protected boolean isPageHistorySortOnCreate() {
		return Boolean.parseBoolean(this.miscProperties.getProperty("page-history-sortoncreate", "true"));
	}

    /**
     * handle any cleanup
     */
    protected void initConversion() {
		this.miscProperties.clear();
	}

	/**
     * Instantiate all the converterStrings
     *
     * @param converterStrings a list of converter strings of the form "key=value"
     * @return a list of converters
     */
    public ArrayList<Converter> createConverters(List<String> converterStrings) {
    	return createConverters(converterStrings, true);
    }
    public ArrayList<Converter> createConverters(List<String> converterStrings, boolean runningState) {
    	String message = "Initializing Converters...";
		if (runningState) this.state.updateNote(message);
		log.info(message);
		
        new DefaultXmlEvents().clearAll(); 	//everytime this method is called, have a clean slate of events
        
        ArrayList<Converter> converters = new ArrayList<Converter>(); 
        this.numNonConverterProperties = 0; 
        for (String converterStr : converterStrings) {
        	if (runningState) this.state.updateProgress();
        	if (runningState && !this.running) {
        		this.feedback = Feedback.CANCELLED;
        		return null;
        	}
            Converter converter;
            if (isNonConverterProperty(converterStr)) {
            	this.numNonConverterProperties++;
            	handleNonConverterProperty(converterStr);
            	continue;
            } 
        	converter = getConverterFromString(converterStr);
        	if (converter == null) {
        		continue;
        	}
        	converters.add(converter);
        }
        if (runningState) addDefaultMiscProperties();
        
        return converters;
    }

	/**
	 * coverts pages with required converters
	 * @param pages
	 */
	protected void convertWithRequiredConverters(List<Page> pages) {
		if (isIllegalHandlingEnabled()) {
			//create pagename converter and convert with it
			String pagenameConvStr = REQUIRED_CONVERTER_ILLEGAL_NAMES;
			ArrayList<Converter> converters = createOneConverter(pagenameConvStr);
			convertPages(pages, converters, "Checking for illegal pagenames.");

			//create linkname converter and convert with it
			String illegallinksConvStr = REQUIRED_CONVERTER_ILLEGAL_LINKS;
			converters = createOneConverter(illegallinksConvStr);
			convertPages(pages, converters, "Checking for links to illegal pagenames.");
		} 
		else {
			log.warn("Illegal Name and Link Handling has been disabled. Skipping required converters.");
		}
    }
    
    /**
     * converts the list of pages with the given converter
     * @param pages list of pages to be converted
     * @param useUI set this to false if you do not want the associated GUI elements
     * to be updated. This is useful for unit testing.
     */
    protected void convertWithRequiredConverters(List<Page> pages, boolean useUI) {
    	//XXX used only by the junit tests
    	
    	//create pagename converter and convert with it
		String pagenameConvStr = REQUIRED_CONVERTER_ILLEGAL_NAMES;
		ArrayList<Converter> converters = createOneConverter(pagenameConvStr);
    	convertPages(pages, converters, "Checking for illegal pagenames.");
    	
    	//get the state hashtable
    	IllegalPageNameConverter  pagenameConverter = (IllegalPageNameConverter) converters.remove(0);
    	HashSet<String> illegalNames =  pagenameConverter.getIllegalPagenames();
    	
    	//create linkname converter and convert with it
		String illegallinksConvStr = REQUIRED_CONVERTER_ILLEGAL_LINKS;
    	converters = createOneConverter(illegallinksConvStr);
    	IllegalLinkNameConverter linknameConverter = (IllegalLinkNameConverter) converters.get(0);
    	linknameConverter.setIllegalPagenames(illegalNames);
    	convertPages(pages, converters, "Checking for links to illegal pagenames.");
    }

	/**
	 * creates the arraylist of converters when only one converter is needed
	 * @param converterString string representing the converter. Should be in property format. Example:<br/>
	 * key=value
	 * @return arraylist with one converter as its sole item
	 */
	protected ArrayList<Converter> createOneConverter(String converterString) {
		ArrayList<String> converterStrings = new ArrayList<String>();
		converterStrings.add(converterString);
		ArrayList<Converter> converters = createConverters(converterStrings);
		return converters;
	}
    
	/**
     * Instantiates a converter from a correctly formatted String.
     * <p/>
     * Note: This method is now only called once per converter -- first all converters
     * are created, then all pages, then all converters are run on all pages.
     *
     * @param converterStr A string of the form "name.keyword=parameters". The
     *  keyword is used to create the correct type of converter, and the parameters
     *  are then passed to the converter. Finally, the "name.keyword" part is set as
     *  the key in the converter, mainly for debugging purposes. 
     * @return converter or null if no converter can be parsed/instantiated
     */
    public Converter getConverterFromString(String converterStr) {
        Converter converter;
        int equalLoc = converterStr.indexOf("=");
        String key = converterStr.substring(0, equalLoc);
        String value = converterStr.substring(equalLoc + 1);
        try {
            if (key.indexOf(CONVERTERTYPE_CLASS) >= 0) {
                converter = getConverterClassFromCache(value);
            } else if (key.indexOf(CONVERTERTYPE_PERL) >= 0) {
                converter = PerlConverter.getPerlConverter(value);
                converter.setValue(value);
            } else if (key.indexOf(CONVERTERTYPE_JAVAREGEXTOKEN) >= 0) {
                converter = JavaRegexAndTokenizerConverter.getConverter(value);
                converter.setValue(value);
            } else if (key.indexOf(CONVERTERTYPE_JAVAREGEX) >= 0) {
                converter = JavaRegexConverter.getConverter(value);
                converter.setValue(value);
            } else if (key.indexOf(CONVERTERTYPE_TWIKICLEANER) >= 0) {
                //converter = getConverterClassFromCache(value);
                converter = TWikiRegexConverterCleanerWrapper.getTWikiRegexConverterCleanerWrapper(value);
                converter.setValue(value);
            } else { 
                String note = "Converter ignored -- name pattern not recognized: " + key;
				this.errors.addError(Feedback.BAD_PROPERTY, note, true);
				log.error(note);
                return null;
            }
            converter.setProperties(this.miscProperties);
        } catch (ClassNotFoundException e) {
            this.errors.addError(Feedback.BAD_CONVERTER_CLASS, "Converter ignored -- the Java class " + value + " was not found", true);
            return null;
        } catch (IllegalAccessException e) {
            this.errors.addError(Feedback.BAD_CONVERTER_CLASS, "Converter ignored -- there was a problem creating a converter object", true);
            return null;
        } catch (InstantiationException e) {
            this.errors.addError(Feedback.BAD_CONVERTER_CLASS, "Converter ignored -- there was a problem creating the Java class " + value, true);
            return null;
        } catch (ClassCastException e) {
            this.errors.addError(Feedback.BAD_CONVERTER_CLASS, "Converter ignored -- the Java class " + value +
                    " must implement the " + Converter.class.getName() + " interface!", true);
            return null;
        }
        converter.setKey(key);
        return converter;
    }

    /**
     * handles necessary state changes for expected properties 
     * that were set in the converter properties file.
     * expected nonconverter properties include hierarchy builder properties
     * and page history preservation properties
     * @param converterStr should be a line from the converter properties file
     * Example:
     * MyWiki.0001.someproperty.somepropertytype=setting
     * <br/>
     * where somepropertytype is an expected property type:
     * <br/>
     * NONCONVERTERTYPE_HIERARCHYBUILDER or NONCONVERTERTYPE_PAGEHISTORYPRESERVATION
     * or NONCONVERTERTYPE_ILLEGALHANDLING
     * or NONCONVERTERTYPE_AUTODETECTSPACEKEYS
     * or NONCONVERTERTYPE_FILTERS
     * or NONCONVERTERTYPE_MISCPROPERTIES
     */
    protected void handleNonConverterProperty(String converterStr) {
    	int equalLoc = converterStr.indexOf("=");
        String key = converterStr.substring(0, equalLoc);
        String value = converterStr.substring(equalLoc + 1);
        String parent = "";
        try {
	    	if (key.indexOf(NONCONVERTERTYPE_HIERARCHYBUILDER) >= 0) {
	    		if (isHierarchySwitch(key))
	    			setHierarchyHandler(value);
	    		else {
					parent = HierarchyBuilder.class.getName(); 
		            Class c;
						c = Class.forName(value);
		            HierarchyBuilder hierarchy = (HierarchyBuilder) c.newInstance();
		            hierarchyBuilder = hierarchy;
		            this.hierarchyBuilder.setProperties(this.miscProperties);
	    		}
	    	}
	    	else if (key.endsWith(NONCONVERTERTYPE_PAGEHISTORYPRESERVATION)) {
	    		handlePageHistoryProperty(key, value);
	    	}
	    	else if (key.endsWith(NONCONVERTERTYPE_ILLEGALHANDLING)) {
	    		handleIllegalHandling(key, value);
	    	}
	    	else if (key.endsWith(NONCONVERTERTYPE_AUTODETECTSPACEKEYS)) {
	    		handleAutoDetectSpacekeys(key, value);
	    	}
	    	else if (key.endsWith(NONCONVERTERTYPE_MISCPROPERTIES)) {
	    		handleMiscellaneousProperties(key, value);
	    	}
	    	else if (key.endsWith(NONCONVERTERTYPE_FILTERS)) {
	    		parent = FileFilter.class.getName();
	    		handleFilters(key, value);
	    	}
	    	else if (key.endsWith(NONCONVERTERTYPE_XMLEVENT)) {
	    		handleXmlEvents(key, value);
	    	}
        } catch (ClassNotFoundException e) { 
            String message = "Property ignored -- the Java class " + value + " was not found";
            log.error(message);
			this.errors.addError(Feedback.BAD_PROPERTY, message, true);
        } catch (IllegalAccessException e) {
            String message = "Property ignored -- there was a problem creating the Java class: " + value +
            	".\n" +
            	"Note: A necessary method's permissions were too restrictive. Check the constructor. ";
            log.error(message);
			this.errors.addError(Feedback.BAD_PROPERTY, message, true);
        } catch (InstantiationException e) {
            String message = "Property ignored -- there was a problem creating the Java class " + value +
            	".\n" +
            	"Note: The class cannot be instantiated as it is abstract or is an interface.";
            log.error(message);
			this.errors.addError(Feedback.BAD_PROPERTY, message, true);
        } catch (ClassCastException e) { 
			String message = "Property ignored -- the Java class " + value +
			                    " must implement the " + parent + " interface!";
            log.error(message);
			this.errors.addError(Feedback.BAD_PROPERTY, message, true);
        } catch (IllegalArgumentException e) {
        	String message = "Property ignored -- property value was not in expected format.";
        	log.error(message);
        	this.errors.addError(Feedback.BAD_PROPERTY, message, true);
        }
    }
    
    /**
     * at long last making some performance enhancements
     * here we are creating an object cache which should help a bit
     *
     * @param key A string representing the converter (actually the part after the
     *        equals sign of the converter string).
     * @return
     * @throws ClassNotFoundException
     * @throws IllegalAccessException
     * @throws InstantiationException
     */
    private Converter getConverterClassFromCache(String key) throws ClassNotFoundException, IllegalAccessException, InstantiationException {
        Converter converter = converterCacheMap.get(key);
        if (converter == null) {
            Class c = Class.forName(key);
            converter = (Converter) c.newInstance();
            converterCacheMap.put(key, converter);
        }
        return converter;
    }

    /**
     * creates file filter.
     * If we have no filter values, returns null.
     * If we have at least one filter value, uses the FilterChain class
     * to create FileFilter that will handle all of the filter requirements.
     * There are two types of supported filters: Class filters, and endswith filters.
     * Class filters are fully resolved class names for classes that implement FileFilter.
     * Endswith filters are text strings that the end of the filename must conform to.
     * If there are more than one filter invoked, the following will be used to resolve 
     * which files to accept: Only pages that all class filters accept as long as 
     * any endswith filter accepts as well will be included. (Class filters are ANDed. 
     * Endswith filters are ORed.) Example: If you had two endswiths, and a class: ".txt",
     * ".xml", and NoSvnFilter, then .txt and .xml files that the NoSvnFilter accepts 
     * will be included.
     * @return FileFilter or null
     */
    protected FileFilter createFilter(final String pattern) {
    	Set<String> values = getFilterValues();
		if (pattern != null && !"".equals(pattern)) 
    		values.add(pattern);

    	if (values.isEmpty()) return null;
    	
    	FilterChain chain = new FilterChain(values, this.miscProperties);
    	return chain.getFilter();
    }
    
    /**
     * Creates PageForXmlRpcOld objects for all the files in inputPages.
     *
     * @param filter file filter to be used to filter input pages, or null, if no such filter should be used
     * @param inputPages   A list of files and directories that Pages should be created for.
     * @return A list of PageForXmlRpcOld objects for all files matching the pattern in the settings.
     */
    protected List<Page> createPages(FileFilter filter, List<File> inputPages) {
       	String message = "Initializing Pages...";
		this.state.updateNote(message);
		log.info(message);
    	
        List<Page> allPages = new LinkedList<Page>();

        for (File fileOrDir : inputPages) {
        	this.state.updateProgress();
            List<Page> pages = recurse(fileOrDir, filter);
            setupPages(fileOrDir, pages);
            allPages.addAll(pages);
        }
        
        if (isHandlingPageHistories() && isPageHistorySortOnCreate()) {
        	return sortByHistory(allPages);
        }
        return allPages;
    }

	/**
     * Recurses through a directory structure and adds all files in it matching the filter.
     * Called by createPages.
     *
     * @param fileOrDir A directory or file. Must not be <code>null</code>.
     * @param filter    the filter to use when selecting files
     * @return A list with PageForXmlRpcOld objects for all the matching files in the directory and its subdirectories
     */
    private List<Page> recurse(File fileOrDir, FileFilter filter) {
        assert fileOrDir != null;
        List<Page> result = new LinkedList<Page>();
        if (fileOrDir.isFile()) {									//it's a file AND
        	if (filter == null || filter.accept(fileOrDir)) {		//there's no filter OR the filter accepts the file
        		PageSplitter splitter = getPageSplitter();
        		if (splitter == null)
        			result.add(new Page(fileOrDir));
        		else 
        			result.addAll(splitter.split(fileOrDir));
        	}
            else 
            	log.debug("Filtering out filename: " + fileOrDir.getName());
        } else if (fileOrDir.isDirectory()) {
            File[] files = fileOrDir.listFiles(filter);
            for (File file : files) {
                result.addAll(recurse(file, filter));
            }
        }
        else { //some other problem
        	String message = "Could not find file: '" +
			        			fileOrDir.getAbsolutePath() +
			        			"'.\n" +
			        			"Check existence and permissions.";
			log.warn(message);
			this.errors.addError(Feedback.BAD_FILE, message, true);
        }
        return result;
    }

    private PageSplitter getPageSplitter() {
    	String classname = this.miscProperties.getProperty("pagesplitter", null);
    	if (classname == null) return null;
    	Class c;
		try {
			c = Class.forName(classname);
		} catch (ClassNotFoundException e) {
			log.error("Could not find pagesplitter class named: " + classname, e);
			return null;
		}
		try {
			PageSplitter splitter = (PageSplitter) c.newInstance();
			return splitter;
		} catch (InstantiationException e) {
			log.error("Could not instantiate pagesplitter class named: " + classname, e);
		} catch (IllegalAccessException e) {
			log.error("Pagesplitter class can not legally be accessed: " + classname, e);
		}
		return null;
	}

	/**
     * Set the names of the pages and performs any other setup needed. Called by recurse().
     * If the user selected a directory and this file is inside it, the base directory's
     * path is removed and the rest is used as the page name.
     * <p/>
     * Any directory separators are replaced with the constant CONFLUENCE_SEPARATOR.
     *
     * @param baseDir The directory that the top-level documents are in
     * @param pages A list of pages to set up
     */
    protected void setupPages(File baseDir, List<Page> pages) {
        String basepath = baseDir.getParentFile().getPath() + File.separator;
        int baselength = basepath.length();

        for (Page page : pages) {
            String pagePath = page.getFile().getPath();
            String pageName = getPagename(pagePath.substring(baselength));
            //Strip the file name from the path.
            String path = getPath(pagePath);
            page.setPath(path);
            page.setName(pageName);
            if (isHandlingPageHistoriesFromFilename()) preserveHistory(page, pageName);
        }
    }

	/**
	 * figures out path var for Page based on complete path to page's file
	 * @param pagePath
	 * @return
	 */
	private String getPath(String pagePath) {
		int fileNameStart = pagePath.lastIndexOf(File.separator);
		if (fileNameStart >= 0) {
		    pagePath = pagePath.substring(0, fileNameStart);
		} else {
		    pagePath = "";
		}
		return pagePath;
	}

    /**
     * uses the filename to set the version and name of the given page
     * so that the history is preserved in the conversion. Note:
     * uses the pageHistorySuffix which is set by the handlePageHistoryProperty 
     * method
     * @param page object that will be changed to reflect pagename and version of given filename 
     * @param filename should use the pageHistorySuffix to indicate version and pagename:
     * <br/>
     * if pageHistorySuffix is -#.txt
     * <br/>
     * then filename should be something like: pagename-2.txt
     * @return Page with changed name and version
     * Will return passed page with no changes if:
     * <ul>
     * <li>suffix is null</li>
     * <li> suffix has no numerical indicator (#)</li>
     * </ul>
     */
    protected Page preserveHistory(Page page, String filename) {
    	//get suffix
    	String suffix = getPageHistorySuffix(); 
    	if (suffix == null) {
    		log.error("Error attempting to preserve history: Page history suffix is Null.");
    		return page;
    	}
    	//create regex for filename based on the suffix
    	Matcher hashFinder = hashPattern.matcher(suffix);
    	String suffixReplaceRegex = "";
    	if (hashFinder.find()) {
    		suffixReplaceRegex = hashFinder.replaceAll("(\\\\d+)");
    		suffixReplaceRegex = "(.*)" + suffixReplaceRegex;
    	} 
    	else {
    		log.error("Error attempting to preserve history: Suffix is invalid. Must contain '#'.");
    		return page;
    	}
    	//get the version and name
    	Pattern suffixReplacePattern = Pattern.compile(suffixReplaceRegex);
    	Matcher suffixReplacer = suffixReplacePattern.matcher(filename);
    	if (suffixReplacer.find()) {
    		String pagename = suffixReplacer.group(1);
    		String versionString = suffixReplacer.group(2);
    		int version = Integer.parseInt(versionString);
    		page.setName(pagename); //set name before version so latestversion data is properly set in Page
    		page.setVersion(version);
    	}
    	return page;
	}

	/**
	 * gets the pagename given the pagepath
	 * @param pagePath
	 * @return pagename
	 */
	protected String getPagename(String pagePath) {
		String pageName = "";
		if (hierarchyHandler == HierarchyHandler.DEFAULT ||
				hierarchyHandler == HierarchyHandler.HIERARCHY_BUILDER) {
		    pageName = pagePath.substring(pagePath.lastIndexOf(File.separator) + 1);
		} else if (hierarchyHandler == HierarchyHandler.PAGENAME_HIERARCHIES) {
			String quotedSeparator = Pattern.quote(File.separator);
			pageName = pagePath.replaceAll(quotedSeparator, CONFLUENCE_SEPARATOR);
		}
		return pageName;
	}
	
	/**
	 * converts the given pages with the given converts
	 * @param pages
	 * @param converters
	 * @return true if conversion of all pages succeeded
	 */
	protected boolean convertPages(List<Page> pages, List<Converter> converters) {
		return convertPages(pages, converters, "Converting pages...");
	}
	
	/**
	 * converts the given pages with the given converters
	 * @param pages
	 * @param converters
	 * @param note, message for the progress monitor
	 * @return true if conversion of all pages succeeded
	 */
	protected boolean convertPages(List<Page> pages, List<Converter> converters, String note) {
		boolean result = true;
		this.state.updateNote(note);
		log.info(note);
		
		this.startTotalConvertTime = (new Date()).getTime();
		
		//go through each page
		for (Iterator<Page> iter = pages.iterator(); iter.hasNext();) {
			if (!this.running) {
				this.feedback = Feedback.CANCELLED;
				return false;
			}
			
			Page page = iter.next();

			//some bookkeeping
			long startTimeStamp = conversionBookkeepingNextPage(page);

            //get the file's contents
			
			if (page.getOriginalText() == null || "".equals(page.getOriginalText())) {
	            File file = getFileContents(page);
	            if (file == null) {
	            	iter.remove(); //get rid of this page from the iterator.
	            	continue;
	            }
			} //else we used a PageSplitter to set the original text, so we can go straight to conversion
            
            //convert the page
            convertPage(converters, page);
            //more bookkeeping
            conversionBookkeepingEndThisPage(startTimeStamp);
            if (!this.running) {
        		this.feedback = Feedback.CANCELLED;
        		return false;
        	}
        }
		//still more bookkeeping
		conversionBookkeepingEndAll(pages, converters);
		return result;
		
	}

	/**
	 * make some log entries about the time it took to convert a page
	 * @param startTimeStamp
	 * @return
	 */
	private long conversionBookkeepingEndThisPage(long startTimeStamp) {
		long stopTimeStamp = ((new Date()).getTime());
		log.info("                   time to convert " + (stopTimeStamp - startTimeStamp) + "ms");
		return stopTimeStamp;
	}

	/**
	 * make some log entries regarding the length of time it took to do the entire conversion
	 * @param pages
	 * @param converters
	 */
	private void conversionBookkeepingEndAll(List<Page> pages, List<Converter> converters) {
		long endTotalConvertTime = (new Date()).getTime();
		long totalTimeToConvert = (endTotalConvertTime - startTotalConvertTime)/1000;        
		String baseMessage = "::: total time to convert files: "+ totalTimeToConvert+ " seconds.";
		log.info(baseMessage);
		String message = baseMessage +
						"For " +
						pages.size() +
						" pages and using " +
						converters.size() +
						" converters.";
		totalsFileLog.info(message);
	}

	/**
	 * update the progress monitor and write some log entries for this page
	 * @param page
	 * @return
	 */
	private long conversionBookkeepingNextPage(Page page) {
		long startTimeStamp = ((new Date()).getTime());
		log.info("-------------------------------------");
		log.info("converting page file: " + page.getName());
		if (page.getFile() != null && page.getFile().getName() != null)
			log.debug("original file name: " + page.getFile().getName());
		return startTimeStamp;
	}

	/**
	 * get the file of the given page 
	 * @param page file for this page
	 * @return the file, or return null if not possible
	 */
	private File getFileContents(Page page) {
		File file = page.getFile();
		if (file == null) {
			if (page.getOriginalText() != null && !"".equals(page.getOriginalText())) {
				log.warn("This appears to be a unit test. Continue as for Unit Test.");
				String path = page.getPath();
				if (path == null) path = "";
				file = new File(path);
			}
			else {
		        log.warn("No file was set for page " + page.getName() + ". Skipping page.");
		        return null;
			}
		}
		else if (page.getOriginalText() == null){
		    try {
		    	String pageContents = "";
		    	if (changingEncoding()) {
		    		String encoding = getEncoding();
		    		byte[] pagebytes = FileUtils.getBytesFromFile(file);
		    		try {
						pageContents = new String(pagebytes, encoding);
					} catch (UnsupportedEncodingException e) {
						String baseerror = "Could not encode file with encoding: " + encoding + ".";
						log.error(baseerror + " Using utf-8.");
						this.errors.addError(Feedback.BAD_SETTING, baseerror, true);
						pageContents = new String(pagebytes, "utf-8");
					}
		    	}
		    	else pageContents = FileUtils.readTextFile(file);
		        page.setOriginalText(pageContents);
		    } catch (IOException e) {
		        String message = "Could not read file " + file.getAbsolutePath() + ".\n" +
							        			"Check existence and permissions.";
		        log.error(message);
				this.errors.addError(Feedback.BAD_FILE, message, true);
		        return null;
		    }

			// Save the true source since the original will get modified in convert.
			page.setUnchangedSource(page.getOriginalText());
		}
		return file;
	}

	private boolean changingEncoding() {
		if (this.miscProperties != null)
			return this.miscProperties.containsKey("encoding"); 
		return false;
	}

	private String getEncoding() {
		if (this.miscProperties != null)
			 return this.miscProperties.getProperty("encoding", "utf-8");
		return "utf-8";
	}


	/**
	 * converts one page with the given converters
	 * @param converters list of converters
	 * @param page page object 
	 */
	protected Page convertPage(List<Converter> converters, Page page) {
		if (page.getConvertedText() == null)
			page.setConvertedText(page.getOriginalText()); //in case empty converter list

		for (Converter converter : converters) {
		    try {
		    	this.state.updateProgress();
		    	if (this.settings != null) {
		    		converter.setAttachmentDirectory(this.settings.getAttachmentDirectory());
		    	}
		    	else {
		    		//for backwards compatibility with v2
		    		ConfluenceSettingsForm confSettings = UWCForm2.getInstance().getConfluenceSettingsForm();
		    		converter.setAttachmentDirectory(confSettings.getAttachmentDirectory());
		    	}
		        converter.convert(page);
		        // Need to reset originalText here because each converted expects
		        // to start with the result of previous conversions.
		        page.setOriginalText(page.getConvertedText());
		    } catch (Exception e) {
		        String note = "Exception thrown by converter " + converter.getKey() +
						                " on page " + page.getName() + ". Continuing with next converter.";
		        log.error(note, e);
				this.errors.addError(Feedback.CONVERTER_ERROR, note, true);
		    }
		    if (converter.getErrors().hasErrors()) {
		    	this.hadConverterErrors = true;
		    	this.state.updateNote(converter.getErrors().getFeedbackWindowErrorMessages());
		    }
		}
		
		return page;
	}

    /**
     * Write pages to disk. They are saved to the directory output/output below the
     * current working directory.
     *
     * @param pages The pages to save
     * @param pattern This file name extension is appended
     *        to each page name to create the file name.
     */
    private void savePages(List<Page> pages, String pattern) {
    	String message = "Saving Pages to Filesystem";
		this.state.updateNote(message);
		log.info(message);
        
    	FileUtils.createOutputDirIfNeeded();
        
    	String outputDirName = UWCGuiModel.getOutputDir();
        log.debug("Output Directory = " + outputDirName);

        File outputDir = new File(outputDirName);
        if (!outputDir.exists() && !outputDir.mkdir()) {
            String dirfailMessage = "Directory creation failed for directory " + outputDirName;
            log.error(Feedback.BAD_OUTPUT_DIR + ": " + dirfailMessage);
			this.errors.addError(Feedback.BAD_OUTPUT_DIR, dirfailMessage, true);
        }
        
        for (Page page : pages) {
        	if (!this.running) {
        		this.feedback = Feedback.CANCELLED;
        		return;
        	}
        	this.state.updateProgress();
            String outputFileLoc = outputDirName + File.separator + page.getName() + pattern;
            FileUtils.writeFile(page.getConvertedText(), outputFileLoc);
        }
    }
    
    protected Vector listCollisions(List<Page> pages) {
    	Vector<String> collisions = new Vector<String>();
    	//check to see if "off" property is present
    	if (this.miscProperties != null && 
    			this.miscProperties.containsKey("list-collisions") &&
    			!Boolean.parseBoolean(this.miscProperties.getProperty("list-collisions"))) {
    		log.debug("Namespace Collisions Feature turned off.");
    		return collisions;
    	}
    	//sort
    	Vector<Page> sorted = new Vector<Page>();
    	sorted.addAll(pages);
    	AsciiVersionComparator version = new AsciiVersionComparator();
    	Collections.sort(sorted, version);
    	//look for collisions
    	Page last = new Page(null);
    	last.setName("");
    	last.setPath("");
    	for (int i = 1; i < sorted.size(); i++) {
			Page page1 = (Page) sorted.get(i-1);
			Page page2 = (Page) sorted.get(i);
			log.debug("Checking for collisions: " + page1.getName() + " and " + page2.getName());
			String collision = "";
			//if each page lower cased is the same
			if (getCollisionComparisonString(page1).equals(getCollisionComparisonString(page2))
					&& page1.getVersion() == page2.getVersion() //and same page history version
				) {
				if (getCollisionComparisonString(page1).equals(getCollisionComparisonString(last))) { //already have one for this name
					String latestPath = getPagePath(page2);
					String current = collisions.remove(collisions.size()-1);
					collision = current + ", " + latestPath + page2.getName();
				}
				else { //starting here
					collision = getPagePath(page1) + page1.getName() + ", " +
								getPagePath(page2) + page2.getName();
				}
				collisions.add(collision);
				last = page1;
				String error = "Potential namespace collision detected for pages: " + collision;
				this.getErrors().addError(Feedback.NAMESPACE_COLLISION, 
						error, 
						true);
				this.log.error(error);
			}
		}
    	return collisions;
    }

	/**
	 * @param page
	 * @return the namespace collision comparison string. Either
	 * the pagename, lower cased, or if we're using the auto detect spacekeys
	 * feature, then the page page + the pagename, all lower cased.
	 */
	protected String getCollisionComparisonString(Page page) {
		if (this.autoDetectSpacekeys)
			return (getPagePath(page) + page.getName()).toLowerCase();
		return page.getName().toLowerCase();
	}

	/**
	 * @param page
	 * @return the page's path, including an ending file seperator if one isn't
	 * already there
	 */
	private String getPagePath(Page page) {
		return (page.getPath().endsWith(File.separator)?
				page.getPath():
				page.getPath()+File.separator);
	}
    
    /**
     * Writes the pages to Confluence. If the process takes more than three seconds,
     * a progress monitor will be displayed so that the user can see that something is
     * indeed happening.
     *
     * @param pages The pages to output.
     * @param spacekey space to which the pages will be written
     */
    protected void writePages(List<Page> pages, String spacekey) {
    	String note = "Uploading Pages to Confluence...";
		this.state.updateNote(note);
		log.info(note);
		
        int numUploaded = 0;
        // at last, write the pages to Confluence!
        for (Page page : pages) {
        	this.state.updateProgress();
        	if (!this.running) {
        		this.feedback = Feedback.CANCELLED;
        		return;
        	}
            if (sendPage(page, null, this.settings) == null) continue;
            numUploaded++;
            if ((numUploaded % 10) == 0) {
                String message = "Uploaded " + numUploaded + 
                	" out of " + pages.size() + 
                	" page"+ (numUploaded==1?"":"s") +
                	".";
                this.state.updateNote(message);
                log.info(message);
            }
        }
        
        String message = "Uploaded " + numUploaded + 
		        		" out of " + pages.size() + 
		        		" page"+ (numUploaded==1?"":"s") +
		        		".";
        this.state.updateNote(message);
        log.info(message);
        
        if (this.settings.getUploadOrphanAttachments().equalsIgnoreCase("true"))
        {
        	ArrayList <File> orphanAttachments=findOrphanAttachments(this.settings.getAttachmentDirectory());
        	uploadOrphanAttachments(orphanAttachments);
        }
        
        //attachedFiles is cleared so that if we do another conversion
        //without closing the UWC, it won't think the attachment has already been
        //attached
        this.attachedFiles = null; 
    }

	    
    /******
     * to find all orphan attachment files
     * @param dirName
     * @return
     */
    protected ArrayList<File> findOrphanAttachments(String dirName)
    {
    	ArrayList<File> orphanAttachments=new ArrayList<File>();
    	File directory=new File(dirName);
    	for (File file : directory.listFiles()) {
    		if (file.isDirectory())
    			orphanAttachments.addAll(findOrphanAttachments(file.getAbsolutePath()));
    		else if (file.isFile())
    		{
    			if (alreadyAttached(file.getName()))
    				continue;  
    			orphanAttachments.add(file);
    		}
        }
    	
    	return orphanAttachments;
    	
    	
    }
    /**
     * to upload the orphan attachment files to wiki.
     */
    protected void uploadOrphanAttachments(ArrayList<File> orphanAttachments)
    {
    	if (orphanAttachments == null || orphanAttachments.size() == 0)
    		return;
    	
    	Hashtable pageTable = new Hashtable();
    	pageTable.put("content", "{attachments}");
    	pageTable.put("title", ORPHAN_ATTACHMENTS_PAGE_TITLE); 
    	
    	//create ConfluenceServerSettings object
    	ConfluenceServerSettings confSettings = new ConfluenceServerSettings();
    	confSettings.login = settings.getLogin();
    	confSettings.password = settings.getPassword();
    	confSettings.url = settings.getUrl(); 
    	confSettings.spaceKey = settings.getSpace();
    	confSettings.truststore = settings.getTruststore();
    	confSettings.trustpass = settings.getTrustpass();
    	confSettings.trustallcerts = settings.getTrustall();
    	
    	RemoteWikiBroker broker = RemoteWikiBroker.getInstance();	
     	//check for problems with settings 
    	checkConfluenceSettings(confSettings);
    	//send page
    	String pageId = sendPage(broker, pageTable, confSettings);
     	
    	int total=orphanAttachments.size();
    	int count=0;
    	for (File file : orphanAttachments) {
    		sendAttachment(file, broker, pageId, confSettings);
    		count++;
    		if ((count % 10) == 0) {
                String message = "Uploaded " + count + 
                	" out of " + total + " orphan attachments.";
                this.state.updateNote(message);
                log.info(message);
            }
        }
    	
    	String message = "Uploaded " + total + " orphan attachments.";
		this.state.updateNote(message);
		log.info(message);
    	  	
    }
    
    /****
     * send an attachment file to wiki.
     * 
     * @param file
     * @param broker
     * @param pageId
     * @param confSettings
     * @return the attachment object we sent. used by junit tests
     */
    protected AttachmentForXmlRpc sendAttachment(File file, RemoteWikiBroker broker, String pageId, ConfluenceServerSettings confSettings) {
    	return sendAttachment(new Attachment(file), broker, pageId, confSettings);
    }
    protected AttachmentForXmlRpc sendAttachment(Attachment attachment, RemoteWikiBroker broker, String pageId, ConfluenceServerSettings confSettings)
    {
    	File file = attachment.getFile();
    	AttachmentForXmlRpc attachmentRpc = new AttachmentForXmlRpc();
        if (tooBig(file) || doesNotExist(file)) 
        	return null;
        attachmentRpc.setFileName(attachment.getName()); 
        attachmentRpc.setFileLocation(file.getAbsolutePath());
        attachmentRpc.setContentType(determineContentType(file)); //XXX Note: if the filename is different from the file, the content type determining might be foiled
        attachmentRpc.setComment(attachment.getComment() == null?getAttachmentUploadComment():attachment.getComment());
        String errorMessage = "Couldn't send attachmentRpc " +
        		file.getAbsolutePath() + ". Skipping attachmentRpc.";
        if (usingWebdav()) {
        	String webdavPath = getWebdavPath();
        	sendAttachmentWebdav(broker, pageId, confSettings, attachmentRpc, webdavPath, errorMessage);
        }
        else 
        	sendAttachmentRemoteAPI(broker, pageId, confSettings, attachmentRpc, errorMessage);
        return attachmentRpc;//for junit tests
    }

	private String getAttachmentUploadComment() {
		if (this.miscProperties != null &&
				this.miscProperties.containsKey("attachment-upload-comment")) {
			String comment = this.miscProperties.getProperty("attachment-upload-comment");
			if (comment == null) return DEFAULT_ATTACHMENT_UPLOAD_COMMENT;
			return comment;
		}
		return DEFAULT_ATTACHMENT_UPLOAD_COMMENT;
	}

	protected boolean usingWebdav() {
		if (this.miscProperties != null && 
				this.miscProperties.containsKey("attachments-use-webdav")) {
			return Boolean.parseBoolean(
					this.miscProperties.getProperty("attachments-use-webdav", "false"));
		}
		return false;
	}

	protected String getWebdavPath() {
		if (this.miscProperties != null && 
				this.miscProperties.containsKey("webdav-path")) {
			return this.miscProperties.getProperty("webdav-path", RemoteWikiBroker.WEBDAV_PATH_EARLY);
		}
		return RemoteWikiBroker.WEBDAV_PATH_EARLY;
	}

	private void sendAttachmentWebdav(RemoteWikiBroker broker, String pageId, 
										ConfluenceServerSettings confSettings, 
										AttachmentForXmlRpc attachment, String basepath, 
										String errorMessage) {
		try {
			Map pagesByIdMap = broker.getAllServerPagesMapById(confSettings, confSettings.spaceKey);
			String webdavPath = broker.getWebDAVPagePath(confSettings.url, confSettings.spaceKey, pageId, pagesByIdMap, basepath);
			broker.sendFileViaWebDAV(attachment.getFileLocation(), webdavPath, confSettings.login, confSettings.password);
		} catch (IOException e) {
        	log.error(Feedback.BAD_FILE + ": " + errorMessage, e);
            this.errors.addError(Feedback.BAD_FILE, errorMessage, true);
        } catch (XmlRpcException e) {
        	log.error(Feedback.REMOTE_API_ERROR + ": " + errorMessage, e);
            this.errors.addError(Feedback.REMOTE_API_ERROR, errorMessage, true);
            if (Pattern.matches(".*?You do not have the permissions.*", e.getMessage())) {
            	String noPermissionsError = "User '" + confSettings.login + 
            		"' " +
            		"does not have permission to attach files to space '" + 
            		confSettings.spaceKey +
            		"'.";
            	log.debug(Feedback.USER_NOT_PERMITTED + ": " + noPermissionsError);
            	this.errors.addError(Feedback.USER_NOT_PERMITTED, noPermissionsError, true);
            }
        }
		
	}

	private void sendAttachmentRemoteAPI(RemoteWikiBroker broker, String pageId, ConfluenceServerSettings confSettings, AttachmentForXmlRpc attachment, String errorMessage) {
		try {
            broker.storeAttachment(confSettings, pageId, attachment);
        } catch (IOException e) {
        	log.error(Feedback.BAD_FILE + ": " + errorMessage, e);
            this.errors.addError(Feedback.BAD_FILE, errorMessage, true);
        } catch (XmlRpcException e) {
        	log.error(Feedback.REMOTE_API_ERROR + ": " + errorMessage, e);
            this.errors.addError(Feedback.REMOTE_API_ERROR, errorMessage, true);
            if (Pattern.matches(".*?You do not have the permissions.*", e.getMessage())) {
            	String noPermissionsError = "User '" + confSettings.login + 
            		"' " +
            		"does not have permission to attach files to space '" + 
            		confSettings.spaceKey +
            		"'.";
            	log.debug(Feedback.USER_NOT_PERMITTED + ": " + noPermissionsError);
            	this.errors.addError(Feedback.USER_NOT_PERMITTED, noPermissionsError, true);
            }
        }
	}
    /**
     * Writes a hierarchy of pages to Confluence. The empty nodes (those with page=null) will
     * not be written if there already exists a page in Confluence. If there is no page at the
     * corresponding place in Confluence, an empty page will be created.
     *
     * Like writePages(), this method will show a progress bar if the hierarchy takes more than
     * a few seconds to send to Confluence.
     *
     * @param root The root of the hierarchy. Note: The root node itself will <strong>NOT</strong> be
     *        added to Confluence. All it's children will be added as top-level pages in the space.
     * @param maxProgress used by logging and status messages
     * @param spacekey space to which the pages will be written
     */
    protected void writeHierarchy(HierarchyNode root, int maxProgress, String spacekey) {
    	String message = "Uploading Pages to Confluence...";
		this.state.updateNote(message);
		log.info(message);
		
        int progressNum = 0;
        this.newNodes = 0; //this has to be a field, because we're already returning something; 
        // at last write the pages to Confluence!
        for (HierarchyNode topLevelPage : root.getChildren()) {
			   log.debug("writeHierarchy: toplevelpage = " + topLevelPage.getName());
				log.debug("number of children this toplevelpage has = " + topLevelPage.getChildren().size());
            progressNum = writeHierarchy(topLevelPage, null, progressNum, maxProgress, spacekey);
            if (!this.running) {
        		this.feedback = Feedback.CANCELLED;
        		return;
        	}
        }
    }

    /**
     * This is the recursive part of <code>writeHierarchy</code>. Don't call this directly!
     * Call writeHierarchy(root) instead.
     *
     * @param node The current node in the hierarchy
     * @param parentId The Confluence "page ID" of the parent page
     * @param progress The number of pages that have been converted so far
     *        (used to keep the progress monitor updated)
     * @param spacekey space to which the pages will be written
     * @return The number of pages converted after this node and all its descendants have been added.
     */
    private int writeHierarchy(
    		HierarchyNode node, 
    		String parentId, 
    		int progress, 
    		int maxProgress, 
    		String spacekey) {
    	if (!this.running) {
    		this.feedback = Feedback.CANCELLED;
    		return progress;
    	}
        
    	// First upload the page contained in this node
        Page page = node.getPage();
        // create missing nodes - like a directory that didn't have a corresponding page
        if (page == null) {
            // This node is a "placeholder" because there are pages further down the hierarchy but
            // for some reason this node was not included in the conversion. Create an empty page.
            // Note that this page will only be sent to Confluence if there was no page in place before.
            page = new Page(null);
            page.setName(node.getName());
            page.setOriginalText("");
            page.setConvertedText("");
            page.setPath(node.getName()); //needed by auto-detect spacekeys feature

            String message = "Page '" + page.getName() + "' does not exist. Creating it now.";
			log.info(message);
			this.state.updateNote(message);
            
            this.newNodes++;
            this.state.updateMax(this.state.getMax() + 1);
        }
        
        //upload the page
        String myId = sendPage(page, parentId, this.settings);
        
        //some bookkeeping
        progress++;
        logProgressMessage(progress, maxProgress);
        
        // Then recursively upload all the node's descendants
        for (HierarchyNode child : node.getChildren()) {
            progress = writeHierarchy(child, myId, progress, maxProgress, spacekey);
            if (!this.running) {
        		this.feedback = Feedback.CANCELLED;
        		return progress;
        	}
        }
        
        return progress;
    }

	/**
	 * sends upload progress messages to the feedback window and the log 
	 * @param current current number of uploaded pages
	 * @param max max number of uploaded pages
	 */
	private void logProgressMessage(int current, int max) {
		this.state.updateProgress();
        String message = "Uploaded " + current + " out of " + (max + this.newNodes) + " pages.";
        
        //more visible note if current is divisible by 10 or last page
        if ((current % 10) == 0 || current == (max + this.newNodes)) { 
        	this.state.updateNote(message);
        	log.info(message);
        }
        else { //less visible note for everything else
        	log.debug(message);
        }
	}

    /**
     * sends page, using settings from  the given settings
     * @param page
     * @param parentId
     * @return page id
     * @throws IllegalArgumentException if a confluenceSetting is invalid
     */
    protected String sendPage(Page page, String parentId, UWCUserSettings settings) {
    	//XXX why are we setting these up every page. Most of these are global. 
    	//XXX If we set these up earlier in the process, we could do the checkConfluenceSettings call 
    	//(currently in the next sendPage) earlier in the process as well
    	ConfluenceServerSettings confSettings = new ConfluenceServerSettings(); 
    	confSettings.login = settings.getLogin();
    	confSettings.password = settings.getPassword();
    	confSettings.url = settings.getUrl(); 
    	confSettings.spaceKey = settings.getSpace();
    	confSettings.truststore = settings.getTruststore();
    	confSettings.trustpass = settings.getTrustpass();
    	confSettings.trustallcerts = settings.getTrustall();
    	
    	//check to see if we've assigned a space to the page
    	if (page.getSpacekey() != null && !"".equals(page.getSpacekey())) { 
    		confSettings.spaceKey = page.getSpacekey();
    		String[] spacedata = page.getSpaceData(page.getSpacekey());
    		String spacename = (spacedata == null || spacedata.length < 1)?page.getSpacekey():spacedata[0];
    		String spacedesc = (spacedata == null || spacedata.length < 2)?"":spacedata[1];
    		if (!createSpace(confSettings, spacename, spacedesc, page.isPersonalSpace(), page.getPersonalSpaceUsername())) {
    			log.warn("Could not create space '" + confSettings.spaceKey + "' assigned to page '" + page.getName() + "'. " +
    					"Using default space from settings.");
    			confSettings.spaceKey = settings.getSpace();
    		}
    	} // check to see if we're automatically detecting spaces based on the file system
    	else if (isAutoDetectingSpacekeys()) { 
    		confSettings.spaceKey = determineSpaceKey(page);
    		if ("".equals(confSettings.spaceKey) || confSettings.spaceKey == null) {
    			String error = "Could not find spacekeys. Note: the auto-detect spacekeys" +
    					" framework is being used. You must choose directories not individual files" +
    					" for conversion.\nCannot upload files to Confluence. Exiting.";
    			log.error(error);
    			this.errors.addError(Feedback.BAD_SPACE, error, true);
    			this.state.updateProgress(this.state.getMax());
    			this.running = false;
    			return "";
    		}
    		if (!createSpace(confSettings)) return null;
    	} //else otherwise use the default (settings based) spacekey
    		
    	return sendPage(page, parentId, confSettings);
    }
    
    /**
     * creates a space for the spacekey in the given settings, if it
     * doesn't already exist
     * @param confSettings
     * @return false if space could not be created
     */
    protected boolean createSpace(ConfluenceServerSettings confSettings) {
    	String spaceName = confSettings.spaceKey;
    	String description = "This space was auto-generated by the UWC.";
    	return createSpace(confSettings, spaceName, description, false, null);
    }
    /**
     * creates a space for the spacekey in the given settings, if it doesn't already exist
     * @param confSettings 
     * @param name name of space to be created. will not be used, if space already exists
     * @param description description of space to be created. will not be used if space already exists
     * @param isPersonalSpace true if space is personal space
     * @param personalSpaceUsername should be non-null if isPersonalSpace is true. If this condition is not
     * maintained, will use spacekey in confSettings instead
     * @return false if space could not be created
     */
    protected boolean createSpace(ConfluenceServerSettings confSettings, String name, String description, 
    		boolean isPersonalSpace, String personalSpaceUsername) {
    	String spacekey = confSettings.spaceKey;
    	RemoteWikiBroker broker = RemoteWikiBroker.getInstance();
    	SpaceForXmlRpc space = broker.createSpace(spacekey, name, description);
    	if (isPersonalSpace) {
    		space.setType(SpaceType.PERSONAL);
    		space.setUsername(personalSpaceUsername);
    	}

    	try {
    		// Conf 2.x will throw an exception if the space doesn't exist.
    		SpaceForXmlRpc space2 = broker.getSpace(confSettings, spacekey); 
    		// Conf 3.x will not throw an exception but will return null
    		if (space2 == null) { //Confluence 3.x
    			String note = "Creating space with spacekey '" + spacekey + "' and name: " + name;
				log.info(note);
    			this.state.updateNote(note);
    			try {
    				broker.addSpace(confSettings, space);
    			} catch (Exception e) {
        			getErrors().addError(Feedback.BAD_LOGIN, 
        					"Could not create space: " + spacekey +
        					" with login: " + confSettings.login +
        					". That login may not have permission to create spaces.", 
        					true);
        			e.printStackTrace();
        			return false;
    			}
    		}
    	} catch (Exception e) { //Confluence 2.x
    		try { //exception! So, try adding the space
    			String note = "Creating space with spacekey '" + spacekey + "' and name: " + name;
				log.info(note);
    			this.state.updateNote(note);
    			broker.addSpace(confSettings, space);
    		} catch (Exception e1) { //something bad happened.
    			getErrors().addError(Feedback.BAD_LOGIN, 
    					"Could not create space: " + spacekey +
    					" with login: " + confSettings.login +
    					". That login may not have permission to create spaces.", 
    					true);
    			e1.printStackTrace();
    			return false;
    		}
    	}
    	return true;
	}

	/**
	 * determines the correct spacekey for the given page.
	 * Note: used in conjunction with the autoDetectSpacekeys Framework
	 * @param page
	 * @return spacekey
	 */
	protected String determineSpaceKey(Page page) {
		log.debug("determining space key. page = " + page.getName());
		String path = page.getPath();
		log.debug("determining space key. path = " + path);
		String ignore = this.miscProperties.getProperty("auto-detect-ignorable-ancestors", "");
		if (this.miscProperties.containsKey("filepath-hierarchy-ignorable-ancestors") &&
				page.getName().equals(path)) {
			ignore = ""; //turn off the ignore property - this is the top level directory
		}
		if (!"".equals(ignore)) {
			log.debug("Ignoring these ancestors: " + ignore);
			path = page.getFile().getParentFile().getAbsolutePath().
					replaceFirst("^\\Q" + ignore + "\\E", "");
		}
		String validated = validateSpacekey(path);
		log.debug("validated space key = " + validated);
		return validated;
	}

	/**
	 * validates the given spacekey, removing illegal chars,
	 * as necessary 
	 * @param spacekey
	 * @return valid spacekey
	 */
	public static String validateSpacekey(String spacekey) {
		String validated = spacekey.replaceAll("[^A-Za-z0-9]", "");
		return validated;
	}

	/*********
	 * to send a page table which has all the attributes for a page to wiki
	 * @param broker
	 * @param pageTable
	 * @param confSettings
	 * @return
	 */
	protected String sendPage(RemoteWikiBroker broker, Hashtable pageTable, ConfluenceServerSettings confSettings)
	{
		PageForXmlRpc brokerPage = PageForXmlRpc.create(pageTable);
		PageForXmlRpc newPage = null;
		try {
			newPage = broker.storeNewOrUpdatePage(confSettings, confSettings.spaceKey, brokerPage);
			if (newPage == null || newPage.getPageParams() == null) {
				String message = "Unknown problem occured while sending page '" + pageTable.get("title") +
				"'. See atlassian-confluence.log for more details.";
				log.error(message);
				getErrors().addError(Feedback.REMOTE_API_ERROR, message, true);
				return null;
			}
		} catch (Exception e) {
			getErrors().addError(Feedback.REMOTE_API_ERROR, 
					"The Remote API threw an exception when it tried to upload page: \"" +
					pageTable.get("title") +
					"\".", true);
			e.printStackTrace();
			return null;
		}	
		
		//move the page if necessary and you can
		String parentid = null;
		if (pageTable.containsKey("parentId"))
			parentid = (String) pageTable.get("parentId");
		else { //We would like to put pages, by default, under the homepage.  
			if (this.homepages.containsKey(confSettings.spaceKey)) {//is there a known Home page in the space?
				parentid = this.homepages.get(confSettings.spaceKey);
				if ("-1".equals(parentid)) parentid = null;// no homepage in this space
			}
			else { //can we find the home page for this space?
				try {
					parentid = broker.getPageIdFromConfluence(confSettings, confSettings.spaceKey, "Home");
					this.homepages.put(confSettings.spaceKey, parentid);
				} catch (Exception e) {
					parentid = null;
					this.homepages.put(confSettings.spaceKey, "-1");
				}
			}
			
		}
		if (parentid != null) {
			try {
				broker.movePage(confSettings, newPage.getId(), parentid, RemoteWikiBroker.Position.APPEND);
			} catch (Exception e) {
				log.debug("Could not move page " + pageTable.get("title") + "\n" + e.getMessage() + "\n" +
							e.getStackTrace()); //could be because Confluence is earlier than 2.9
			}
		}
		
    	return newPage.getId();
		
	}
	
	/*********
	 * to send a blog table which has all the attributes for a blog to wiki
	 * @param broker
	 * @param pageTable
	 * @param confSettings
	 * @return
	 */
	protected String sendBlog(RemoteWikiBroker broker, Hashtable pageTable, ConfluenceServerSettings confSettings)
	{
		BlogForXmlRpc brokerPage = BlogForXmlRpc.create(pageTable);
		BlogForXmlRpc newPage = null;
		try {
			newPage = broker.storeBlog(confSettings, confSettings.spaceKey, brokerPage);
			if (newPage == null || newPage.getblogParams() == null) {
				String message = "Unknown problem occured while sending page '" + pageTable.get("title") +
				"'. See atlassian-confluence.log for more details.";
				log.error(message);
				getErrors().addError(Feedback.REMOTE_API_ERROR, message, true);
				return null;
			}
		} catch (Exception e) {
			getErrors().addError(Feedback.REMOTE_API_ERROR, 
					"The Remote API threw an exception when it tried to upload page: \"" +
					pageTable.get("title") +
					"\".", true);
			e.printStackTrace();
			return null;
		}	
    	return newPage.getId();
	}
	
	
	protected void checkConfluenceSettings(ConfluenceServerSettings confSettings)
	{
		//check for problems with settings 
    	Feedback testConnectionFeedback = TestSettingsListener.getConnectionFeedback(confSettings, isAutoDetectingSpacekeys());
    	if (testConnectionFeedback != Feedback.OK) {
    		String message = TestSettingsListener.getConnectionFeedbackMessage(confSettings, isAutoDetectingSpacekeys());
    		log.error(message);
    		this.state.updateNote(message);
    		throw new IllegalArgumentException(message);
    	}
//    	log.info(TestSettingsListener.SUCCESS_MESSAGE_LONG); //this is getting called for every page 
		
	}
	/**
     * sends page using the given settings
     * @param page
     * @param parentId
     * @param confSettings
     * @return page id the confluence page id for the page being stored. Used if
     * page is new.
     */
    protected String sendPage(Page page, String parentId, ConfluenceServerSettings confSettings) {
    	//create wiki broker
    	RemoteWikiBroker broker = RemoteWikiBroker.getInstance();
    	//create page that broker can use
    	Hashtable pageTable = createPageTable(page, parentId);
     	//check for problems with settings 
    	checkConfluenceSettings(confSettings); //XXX Why are we doing this for every page? 'cause we seem to create the confSettings on a page by page basis?
    	//send page
    	String id = null;
    	if (page.isBlog()) {
    		log.debug("Attempting to send blog: " + page.getName() + " to space: " + confSettings.spaceKey);
    		id = sendBlog(broker, pageTable, confSettings);
    	} else { 
    		log.debug("Attempting to send page: " + page.getName() + " to space: " + confSettings.spaceKey);
    		id = sendPage(broker, pageTable, confSettings);
    	}
    	if (id == null) return null;
     	//send attachments
    	sendAttachments(page, broker, id, confSettings);
    	//send labels 
    	sendLabels(page, broker, id, confSettings);
    	//send comments
    	sendComments(page, broker, id, confSettings);
    	//set author
    	sendAuthor(page, broker, id, confSettings);
    	//set timestamp
    	sendTimestamp(page, broker, id, confSettings);
    	//return the page id
    	return id;
    }


	/**
     * creates a parameter table with the given page and parentId.
     * @param page
     * @param parentId
     * @return table with Remote API page parameters
     */
    private Hashtable createPageTable(Page page, String parentId) {
    	Hashtable table = new Hashtable();
    	table.put("content", page.getConvertedText());
    	table.put("title", page.getName()); 
    	if (parentId != null && !parentId.equals("null")) table.put("parentId", parentId);
    	if (page.getVersion() > 0) table.put("version", page.getVersion() + "");
    	return table;
	}

    /**
     * checks the given page for valid attachments,
     * and sends them to Confluence using the rwb and the given pageId string as the 
     * page to be attached to.
     * 
     * @param page given page object that might have attachments
     * @param broker XML-RPC broker which will communicate with Confluence
     * @param pageId the page the Confluence will attach the attachment to
     * @param confSettings the confluence user settings needed by the broker to connect to Confluence 
     */
    private void sendAttachments(Page page, RemoteWikiBroker broker, String pageId, ConfluenceServerSettings confSettings) {
    	log.debug("Examining attachments for page: " + page.getName());
		// Send the attachments
        for (Attachment attachment : page.getAllAttachmentData()) {
        	if (alreadyAttached(page, attachment.getFile()))
        		continue;
        	sendAttachment(attachment, broker, pageId, confSettings);
        }
	}
    
    protected void sendLabels(Page page, RemoteWikiBroker broker, String pageId, ConfluenceServerSettings confSettings) {
    	log.debug("Examining labels for page: " + page.getName());
    	//check to see if we're sending labels for this version of the page
		if (badVersionForSendingLabels(page)) return;
    	String labels = page.getLabelsAsString();
    	log.debug("Sending Labels: " + labels);
    	if (labels == null) 
    		return;
    	try {
    		broker.addLabels(confSettings, labels, pageId);
    	} catch (Exception e) {
    		String errorMessage = "Could not add labels '" + labels + "' to page '" + page.getName() +"'";
    		log.error(Feedback.REMOTE_API_ERROR + ": " + errorMessage);
    		this.errors.addError(Feedback.REMOTE_API_ERROR, errorMessage, true);
    	}
    }

	private boolean badVersionForSendingLabels(Page page) {
		int version = page.getVersion();
    	int latest = Page.getLatestVersion(page.getName());
    	boolean history = isHandlingPageHistories();
    	String allVersionsProp = (String) this.miscProperties.get("page-history-allversionlabels");
    	boolean allVersions = (allVersionsProp != null) && Boolean.parseBoolean(allVersionsProp);
		return history && !allVersions && version != latest;
	}
    
    /**
     * adds page comments to a page in confluence
     * @param page given page object that might have comments
     * @param broker XML-RPC broker which will communicate with Confluence
     * @param pageId the page id for the given page
     * @param confSettings the confluence user settings needed by the broker to connect to Confluence
     */
    protected void sendComments(Page page, RemoteWikiBroker broker, String pageId, ConfluenceServerSettings confSettings) {
    	if (page.hasComments()) {
    		log.debug("Sending comments for page: " + page.getName());
    		try {
    			for (Comment comment : page.getAllCommentData()) {
    				//create page that broker can use
    				CommentForXmlRpc brokerComment = new CommentForXmlRpc();
    				brokerComment.setPageId(pageId);
    				brokerComment.setContent(comment.text);
    				//upload comment
    				CommentForXmlRpc uploadedComment = broker.addComment(confSettings, brokerComment);
    				if (comment.hasCreator()) {
    					boolean usersMustExist = false;
						broker.setCreator(confSettings, comment.creator, uploadedComment.getId(), usersMustExist);
						broker.setLastModifier(confSettings, comment.creator, uploadedComment.getId(), usersMustExist);
    				}
    				if (comment.hasTimestamp()) {
    					broker.setCreateDate(confSettings, comment.timestamp, uploadedComment.getId());
    					broker.setLastModifiedDate(confSettings, comment.timestamp, uploadedComment.getId());
    				}
    			}
    		} catch (Exception e) {
    			String errorMessage = null;
    			if (e.getMessage().contains("NotPermittedException")) {
    				errorMessage = "User is not permitted to add comments to page: " + page.getName() + "'";
    			}
    			else if (e.getMessage().contains("does not exist")) {
    				errorMessage = "Cannot add comments to the page because it does not exist: " + page.getName();
    			}
    			else {
    				errorMessage = "Could not send comments to page '" + page.getName() +"'";
    			}
    			log.error(Feedback.REMOTE_API_ERROR + ": " + errorMessage);
    			this.errors.addError(Feedback.REMOTE_API_ERROR, errorMessage, true);
    		}
    	}
    	else log.debug("Page has no comments."); //DELETE
    }
	private void sendAuthor(Page page, RemoteWikiBroker broker, String id, ConfluenceServerSettings confSettings) {
		if (page.getAuthor() != null) {
			log.debug("Sending author data.");
			boolean exist = true;
			if (this.miscProperties.containsKey("users-must-exist")) 
				exist = Boolean.parseBoolean((String) this.miscProperties.get("users-must-exist"));
			try {
				if (page.getVersion() == 1) { //only set the creator if its the first version
					broker.setCreator(confSettings, page.getAuthor(), id, exist);
				}
				//set the modifier 
				broker.setLastModifier(confSettings, page.getAuthor(), id, exist);
			} catch (Exception e) {
				String errorMessage = Feedback.REMOTE_API_ERROR + ": Problem setting creator or last modifier data.";
				log.error(errorMessage);
    			this.errors.addError(Feedback.REMOTE_API_ERROR, errorMessage, true);
				e.printStackTrace();
			}
		}
	}
	
	private void sendTimestamp(Page page, RemoteWikiBroker broker, String id, ConfluenceServerSettings confSettings) {
		if (page.getTimestamp() != null) {
			log.debug("Sending timestamp data.");
			try {
				DateFormat dateFormat = new SimpleDateFormat("yyyy:MM:dd:HH:mm:ss:SS"); //XXX Settable?
			    String timestamp = dateFormat.format(page.getTimestamp());

				if (page.getVersion() == 1) { //only set the creator if its the first version
					broker.setCreateDate(confSettings, timestamp, id);
				}
				//set the modifier 
				broker.setLastModifiedDate(confSettings, timestamp, id);
			} catch (Exception e) {
				String errorMessage = Feedback.REMOTE_API_ERROR + ": Problem setting create or last modified date.";
				log.error(errorMessage);
    			this.errors.addError(Feedback.REMOTE_API_ERROR, errorMessage, true);
				e.printStackTrace();
			}
		}
	}

    
    /**
     * @param file
     * @return true if the given file does not exist on the filesystem.
     */
    protected boolean doesNotExist(File file) {
		boolean doesNotExist = !file.exists();
		if (doesNotExist)
			log.warn("File '" + file.getPath() + "' does not exist: Skipping");
		return doesNotExist;
	}

	/**
     * @param file
     * @return true if file size is too big
     */
    protected boolean tooBig(File file) {
    	
    	if (!file.exists()) return false;
		int length = (int) file.length();
		
		String maxString = getMaxAttachmentSizeString();
		
		int maxBytes = getAsBytes(maxString);
		if (maxBytes < 0) return false;
		boolean tooBig = length > maxBytes;
		if (tooBig)
			log.warn("File " + file.getName() + " is larger than " + maxString + ". Skipping.");
		return tooBig;
	}

	/**
	 * @return the max attachment size as a string
	 */
	private String getMaxAttachmentSizeString() {
		return getMaxAttachmentSizeStringFromModel();
	}

	/**
	 * @return the max attachment size as it's represented in the model
	 */
	private String getMaxAttachmentSizeStringFromModel() {
		return this.settings.getAttachmentSize();
	}

	/**
     * @param fileLocation location of properties file
	 * @return a Properties object containing confluence settings
     */
    protected Properties loadProperties(String fileLocation) {
    	/* most of this code grabbed from ConfluenceSettingsForm.populateConfluenceSettings*/
    	Properties properties = new Properties();
        File confSettings = new File(fileLocation);
        if (confSettings.exists()) {
            // load properties file
            FileInputStream fis;
            try {
                fis = new FileInputStream(fileLocation);
				properties.load(fis);
                fis.close();
            } catch (FileNotFoundException e) {
                e.printStackTrace();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        return properties;
    }

    /**
     * calculates the number of bytes the given maxString represents
     * @param maxString file size described as a String.
     * Example: 5B, 5K, 5M, 5G, etc.
     * @return as Bytes.
     * Respectively: 5, 5120, 5242880, 5368709120
     */
    protected int getAsBytes(String maxString) {
    	String maxRegex = "^(\\d+)(\\D)";
    	if (maxString == null || "".equals(maxString)) 
    		return -1;
    	
    	int power, num = 0;
    	String numString, unitString = null;
    	if (Pattern.matches("^\\d+$", maxString)) {
    		unitString = "B";
    		numString = maxString;
    	}
    	else {
	    	numString = maxString.replaceFirst(maxRegex, "$1");
	    	unitString = maxString.replaceFirst(maxRegex, "$2");
    	}
    	try {
    		num = Integer.parseInt(numString);
    	} catch (NumberFormatException e) {
    		String message = PROP_ATTACHMENT_SIZE_MAX + " setting is malformed.\n" +
    				"Setting must be formatted like so: [number][unit], where unit is\n" +
    				"one of the following: B, K, M, G. No max attachment size set.";
    		log.error(message);
    		return -1;
    	}
    	unitString = unitString.toUpperCase();
    	char unit = unitString.toCharArray()[0]; //first char in that string
		
    	switch (unit) {
			case ('B'): power = 0;break;
			case ('K'): power = 1;break;
			case ('M'): power = 2;break;
			case ('G'): power = 3;break;
			default: return -1;
		}
		
		int multiplier = (int) Math.pow(1024, power);
		int value = num * multiplier;
		return value;
	}

	/**
     * @param page
     * @param file
     * @return true if a particular page already has a particular
     * file attached.
     */
    protected boolean alreadyAttached(Page page, File file) {
		String pagename = page.getName();
		String filename = file.getName();
		String attachmentId = pagename + filename;
		if (attachedFiles == null) 
			attachedFiles = new HashSet<String>();
		boolean attached = attachedFiles.contains(attachmentId);
		
		if (!attached) attachedFiles.add(attachmentId);
		else log.debug("Attachment " + filename + " is already attached: Skipping.");
		
		return attached;
	}
    
    /**
     * to check if the file has been attached with any pages.
     * @param fileName
     * @return
     */    
    protected boolean alreadyAttached(String fileName) {
    	if (this.attachedFiles == null || this.attachedFiles.isEmpty())
    		return false;
    	Iterator <String>it=attachedFiles.iterator();
    	while (it.hasNext())
    	{
    		String item=it.next();
    		int index=item.lastIndexOf(fileName);
    		if (index < 0)
    			continue;
    		if (item.length() - index == fileName.length())
    			return true;		
    	}
    	return false;
    
	}

    /**
     * This method determines the mime type of a file. It uses the file
     * conf/mime.types to map from the file name extension
     * to a mime type. The mime type file should be read into the
     * mimeTypes field before this method is called.
     *
     * @param file The file object
     * @return the mime type of the file.
     */
    public static String determineContentType(File file) {
        if (mimeTypes != null) {
            return mimeTypes.getContentType(file);
        } 
        //else assume it's an image
        String filename = file.getName();
        int extensionStart = filename.lastIndexOf(".");
        if (extensionStart >= 0) {
            String extension = filename.substring(extensionStart + 1);
            return "image/" + extension;
        }
        // Hmm... No extension. Assume it's a text file.
        return "text/plain";
    }
    
    Pattern switchPattern = Pattern.compile("switch");
    Pattern suffixPattern = Pattern.compile("suffix");
    private boolean handlingPageHistories = false;
	private String pageHistorySuffix = null;
    
    /**
     * set the page history state to reflect the page history property
     * and associated value that are passed as arguments
     * @param key
     * @param value
     */
    protected void handlePageHistoryProperty(String key, String value) {
    	Matcher switchFinder = switchPattern.matcher(key);
    	if (switchFinder.find()) {
    		//the default should be false, so it's ok to just parse the string.
    		this.handlingPageHistories = Boolean.parseBoolean(value);
    		return;
    	}
    	Matcher suffixFinder = suffixPattern.matcher(key);
    	if (suffixFinder.find()) {
    		setPageHistorySuffix(value);
    		return;
    	}
    }
    
    protected void handleIllegalHandling(String key, String value) {
    	boolean enabled = true; //default
    	value = value.trim();
    	if ("false".equals(value))
    		enabled = false;
    	illegalHandlingEnabled = enabled;
    }
    
    protected void handleAutoDetectSpacekeys(String key, String value) {
    	boolean enabled = false; //default
    	value = value.trim();
    	if ("true".equals(value)) {
    		enabled = true;
    	}
    	autoDetectSpacekeys = enabled;
    }
    
    Pattern miscPropsPattern = Pattern.compile("" +
    		"\\w+\\.\\d+\\.([^.]+)\\.property"
    		);
    protected Properties handleMiscellaneousProperties(String key, String value) {
    	Matcher miscKeyFinder = miscPropsPattern.matcher(key);
    	if (miscKeyFinder.matches()) {
    		String misckey = miscKeyFinder.group(1);
    		if (this.miscProperties == null)
    			this.miscProperties = new Properties();
    		this.miscProperties.put(misckey, value);
    		log.debug("Miscellaneous Property set: " + misckey + "=" + value);
    		return this.miscProperties;
    	}
    	String error = "Miscellaneous property was detected, " +
		    			"but key was invalid. Could not instantiate property: " +
		    			key + "=" + value;
		log.error(error);
		this.errors.addError(Feedback.BAD_PROPERTY, error, true);
		return this.miscProperties;
    }
    
    private void addDefaultMiscProperties() {
    	handleMiscellaneousProperties("Testing.1234.spacekey.property", this.settings.getSpace());
    }
    
    protected void handleFilters(String key, String value) throws InstantiationException, IllegalAccessException {
    	log.debug("filter property = " + value);
    	getFilterValues().add(value);
    }

	private Set<String> getFilterValues() {
		if (this.filterValues == null)
			this.filterValues = new HashSet<String>();
		return this.filterValues;
	}
    
    /**
     * sets up .xmlevent properties
     * @param key must end in .xmlevent
     * @param value must follow this format:
     * <tt>
     * {tag}tagname{class}classname
     * </tt>
     * where tagname is the xml tag to associate the event with (b, for bold)
     * and classname is the parser that will manage the events for that tag.
     * tagname can contain a comma-delimited list of tags. For example:
     * {tag}h1, h2, h3{class}com.example.HeaderParser
     */
    private void handleXmlEvents(String key, String value) {
    	String tag = getXmlEventTag(value);
    	String classname = getXmlEventClassname(value);
    	String[] tags = tag.split(",");
    	for (String onetag : tags) {
    		onetag = onetag.trim();
			addOneXmlEvent(onetag, classname);
		}
    }

	/**
	 * adds one xml event object to the events handler, such that the classname becomes 
	 * an instantiated class that is associated with the given tag.
	 * The events handler can be custom (using the xmlevents misc property), or the default
	 * xml events handler will be used
	 * @param tag
	 * @param classname
	 */
	private void addOneXmlEvent(String tag, String classname) {
		if (this.miscProperties.containsKey("xmlevents")) {
    		Class eventsClass;
    		String xmleventsclass = this.miscProperties.getProperty("xmlevents");
			try {
				eventsClass = Class.forName(xmleventsclass);
			} catch (ClassNotFoundException e) {
				log.warn("xmlevents property value - " +
						xmleventsclass +
						" - does not exist. Using DefaultXmlEvents.");
				this.miscProperties.remove("xmlevents");
				eventsClass = DefaultXmlEvents.class; //try setting the DefaultXmlEvents
			}
    		XmlEvents events = null;
			try {
				events = (XmlEvents) eventsClass.newInstance();
				events.addEvent(tag, classname); //call the custom events class
				return;
			} catch (Exception e) { 
				log.warn("xmlevents property value - " +
						xmleventsclass + " - hasn't implemented XmlEvents. " +
						"Using DefaultXmlEvents.");
				this.miscProperties.remove("xmlevents");
				//continue to DefaultXmlEvents.addEvent below
			}
    	}
		new DefaultXmlEvents().addEvent(tag, classname);
	}
    
    Pattern xmleventClassPattern = Pattern.compile("" +
    		"\\{class\\}(.*)");
    protected String getXmlEventClassname(String value) {
		Matcher finder = xmleventClassPattern.matcher(value);
		if (finder.find()) {
			return finder.group(1);
		}
		throw new IllegalArgumentException(XMLEVENT_PROP_ERROR);
	}

    Pattern xmleventTagPattern = Pattern.compile("" +
    		"\\{tag\\}([^}]+)\\{class\\}");
	protected String getXmlEventTag(String value) {
		Matcher finder = xmleventTagPattern.matcher(value);
		if (finder.find()) {
			return finder.group(1);
		}
		throw new IllegalArgumentException(XMLEVENT_PROP_ERROR);
	}

	/**
     * @param key
     * @return true if the given key is the switch to turn on the 
     * Hierarchy framework
     */
    protected boolean isHierarchySwitch(String key) {
    	Matcher switchFinder = switchPattern.matcher(key);
    	return switchFinder.find();
    }
    
    /**
     * determines if the given string represents an allowed 
     * non converter property: (hierarchy builder, page history preserver,
     * illegalname handler, autodetect spacekeys)
     * @param input represents an entire converter/property string. For example:
     * <br/>
     * Wiki.0011.somefilename.propertytype=something
     * @return true if it's an expected/allowed non converter property
     */
    public boolean isNonConverterProperty(String input) {
    	String converterTypes = 
    				"(" +
    					"(" + 
    						NONCONVERTERTYPE_HIERARCHYBUILDER + 
    					")" + 
    					"|" +
    					"(" + 
    						NONCONVERTERTYPE_PAGEHISTORYPRESERVATION + 
    					")" +
    					"|" +
    					"(" + 
							NONCONVERTERTYPE_ILLEGALHANDLING + 
						")" +
    					"|" +
    					"(" + 
							NONCONVERTERTYPE_AUTODETECTSPACEKEYS + 
						")" +
						"|" +
						"(" +
							NONCONVERTERTYPE_FILTERS + 
						")" +
						"|" +
						"(" +
							NONCONVERTERTYPE_MISCPROPERTIES +
						")" +
						"|" +
						"(" +
							NONCONVERTERTYPE_XMLEVENT +
						")" +
    				")";
    	String converterPattern = "[-\\w\\d.]+?" + converterTypes + "=" + ".*";
    	return input.matches(converterPattern);
    }
    /**
     * @return true if the converter should handle page histories
     */
    public boolean isHandlingPageHistories() {
    	return this.handlingPageHistories;
    }
    
    /**
     * @return true if the converter should handle page histories
     */
    public boolean isHandlingPageHistoriesFromFilename() {
    	return this.handlingPageHistories && this.pageHistorySuffix != null;
    }
    
    /**
     * @return the current page history suffix
     */
    public String getPageHistorySuffix() {
    	return this.pageHistorySuffix;
    }
    
	/**
	 * sorts the given list of pages.
	 * Note: sorting will take into account page name 
	 * and page version. Non unique page objects will be culled.
	 * @param pages list of Page objects
	 * @return sorted list
	 */
	protected List<Page> sortByHistory(List<Page> pages) {
		this.state.updateNote("Sorting Pages by Page History");
		List<Page> sortedPages = new ArrayList<Page>();
		Set<Page> sorted = new TreeSet<Page>();
		sorted.addAll(pages); //sort them and get rid of non-unique pages
		sortedPages.addAll(sorted); //turn them back into a list
		return sortedPages;
	}

	Pattern hashPattern = Pattern.compile("#+");
	/**
	 * sets the page history suffix, if it's a valid suffix.
	 * If not, sets it to null.
	 * @param suffix candidate suffix, a valid candidate will have 
	 * a numeric component, represented by a '#' (hash) symbol
	 * <br/>
	 * Example: -v#.txt
	 * @return true, if a valid suffix was saved.
	 * false, if the suffix was invalid, and therefore was not saved.
	 */
	protected boolean setPageHistorySuffix(String suffix) {
		//check for suffix goodness
		Matcher hashFinder = hashPattern.matcher(suffix);
		if (hashFinder.find()) {
			this.pageHistorySuffix = suffix;
			return true;
		}
		log.error("Error trying to preserve page history: Suffix '" + suffix + "' " +
				"does not have a sortable component. Must include '#'.");
		this.pageHistorySuffix = null;
		return false;
	}

	/**
	 * @return HierarchyBuilder object. used by tests.
	 */
	protected HierarchyBuilder getHierarchyBuilder() {
		return hierarchyBuilder;
	}

	/**
	 * @return HierarchyHandler object. used by tests.
	 */
	protected HierarchyHandler getHierarchyHandler() {
		return hierarchyHandler;
	}
	
	/**
	 * sets how the hierarchy framework is to be used. 
	 * @param input "UseBuilder", "UsePagenames", or "Default". 
	 * If input is none of these, no changes occur
	 */
	private void setHierarchyHandler(String input) {
		if (input.matches("UseBuilder")) hierarchyHandler = HierarchyHandler.HIERARCHY_BUILDER;
		else if (input.matches("UsePagenames")) hierarchyHandler = HierarchyHandler.PAGENAME_HIERARCHIES;
		else if (input.matches("Default")) hierarchyHandler = HierarchyHandler.DEFAULT;
	}

	/**
	 * @return the feedback as it currently stands
	 */
	public Feedback getConverterFeedback() {
		return this.feedback;
	}

	/**
	 * resets the feedback state to Feedback.NONE
	 */
	public void resetFeedback() {
		this.feedback = Feedback.NONE;
	}
	
	/**
	 * clears state relating to the error handling
	 */
	public void resetErrorHandlers() {
		this.errors.clear();
		this.hadConverterErrors = false;
	}
	
	/**
	 * clears state relating to the hierarchy framework
	 */
	public void resetHierarchy() {
		this.hierarchyBuilder = null;
		this.hierarchyHandler = HierarchyHandler.DEFAULT;
	}
	
	/**
	 * @return object contains information relating to errors triggered during the conversion
	 */
	public ConverterErrors getErrors() {
		return this.errors;
	}
	
	/**
	 * @return true if the conversion has generated errors
	 */
	public boolean hadConverterErrors() {
		return this.hadConverterErrors;
	}
	
	/**
	 * setter
	 * @param running
	 */
	protected void setRunning(boolean running) {
		this.running = running; //used in junit
	}
	
	/**
	 * setter
	 * @param settings
	 */
	protected void setSettings(UWCUserSettings settings) {
		this.settings = settings; //used in junit
	}

	
	/**
	 * @return true if the illegal handling (names and links) should occur. 
	 * false if it should be disabled
	 */
	public boolean isIllegalHandlingEnabled() {
		return illegalHandlingEnabled;
	}
	
	public boolean isAutoDetectingSpacekeys() {
		return autoDetectSpacekeys;
	}
	
	public class AsciiVersionComparator implements Comparator {
	      public int compare(Object a, Object b) {
	    	  Page pa = (Page) a;
	    	  Page pb = (Page) b;
	    	  String sa = pa.getName().toLowerCase();
	    	  String sb = pb.getName().toLowerCase();
	    	  int ascii = sa.compareTo(sb);
	    	  int sav = pa.getVersion();
	    	  int sbv = pb.getVersion();
	    	  int version = sbv - sav;
	    	  return ascii - version;
	      }
	}
}
